This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0c77b73d15d59633e4b040d7b8518638ee9d2edc Author: Gavin Chou <[email protected]> AuthorDate: Thu Oct 15 09:19:59 2026 +0800 branch-4.1: [opt](partition) Add global dynamic partition drop protection #68844 (#69065) Cherry-picked from #68844 Conflict resolution: - Adapted dynamic partition cleanup to branch-4.1 `DropPartitionClause` APIs. - Adapted the global variable annotation and tests to branch-4.1 FE APIs/JUnit 4. Validation: - `sh run-fe-ut.sh --run "org.apache.doris.catalog.DynamicPartitionTableTest,org.apache.doris.qe.VariableMgrTest"` (65 tests passed) --- .../doris/clone/DynamicPartitionScheduler.java | 30 +++++-- .../java/org/apache/doris/qe/SessionVariable.java | 17 ++++ .../main/java/org/apache/doris/qe/VariableMgr.java | 30 +++++++ .../doris/catalog/DynamicPartitionTableTest.java | 97 ++++++++++++++++++++++ .../java/org/apache/doris/qe/VariableMgrTest.java | 26 ++++++ 5 files changed, 194 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index a35ec5696bd..e0aa3aeb1b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -59,6 +59,8 @@ import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.meta.MetaContext; import org.apache.doris.nereids.util.DateUtils; import org.apache.doris.persist.PartitionPersistInfo; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.VariableMgr; import org.apache.doris.thrift.TStorageMedium; import com.google.common.annotations.VisibleForTesting; @@ -777,13 +779,17 @@ public class DynamicPartitionScheduler extends MasterDaemon { executeFirstTime); } clearDropPartitionFailedMsg(olapTable.getId()); - if (olapTable.getPartitionRetentionCount() > 0) { - // Handle AUTO PARTITION cleanup based on partition.retention_count - dropPartitionClauses = getDropPartitionClauseForAutoPartition(db, olapTable); + if (VariableMgr.isDynamicPartitionDropProtectionEnabled()) { + logDynamicPartitionDropProtection(db, olapTable); } else { - // Handle dynamic partition cleanup - dropPartitionClauses = getDropPartitionClauseForDynamic(db, olapTable, partitionColumn, - partitionFormat); + if (olapTable.getPartitionRetentionCount() > 0) { + // Handle AUTO PARTITION cleanup based on partition.retention_count + dropPartitionClauses = getDropPartitionClauseForAutoPartition(db, olapTable); + } else { + // Handle dynamic partition cleanup + dropPartitionClauses = getDropPartitionClauseForDynamic(db, olapTable, partitionColumn, + partitionFormat); + } } tableName = olapTable.getName(); } catch (Exception e) { @@ -801,6 +807,10 @@ public class DynamicPartitionScheduler extends MasterDaemon { continue; } try { + if (VariableMgr.isDynamicPartitionDropProtectionEnabled()) { + logDynamicPartitionDropProtection(db, olapTable); + break; + } Env.getCurrentEnv().dropPartition(db, olapTable, dropPartitionClause); } catch (Exception e) { recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId()); @@ -1003,6 +1013,14 @@ public class DynamicPartitionScheduler extends MasterDaemon { createOrUpdateRuntimeInfo(tableId, CREATE_PARTITION_MSG, DEFAULT_RUNTIME_VALUE); } + private void logDynamicPartitionDropProtection(Database db, OlapTable olapTable) { + LOG.warn("Skip automatic partition drop for db [{}-{}], table [{}-{}] because global session variable " + + "'{}' is enabled. Existing partitions are retained; set it to false to resume automatic " + + "partition cleanup", + db.getId(), db.getName(), olapTable.getId(), olapTable.getName(), + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION); + } + private void recordDropPartitionFailedMsg(String dbName, String tableName, String msg, long tableId) { LOG.warn("dynamic drop partition failed: {}, db: {}, table: {}", msg, dbName, tableName); createOrUpdateRuntimeInfo(tableId, DYNAMIC_PARTITION_STATE, State.ERROR.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 43e4937b05a..43a0a23815f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -204,6 +204,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT = "enable_rewrite_element_at_to_slot"; public static final String ENABLE_ODBC_TRANSCATION = "enable_odbc_transcation"; public static final String ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS = "enable_binary_search_filtering_partitions"; + public static final String ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION = + "enable_dynamic_partition_drop_protection"; public static final String CACHE_SORTED_PARTITION_INTERVAL_SECOND = "cache_sorted_partition_interval_second"; public static final String SKIP_PRUNE_PREDICATE = "skip_prune_predicate"; public static final String ENABLE_SQL_CACHE = "enable_sql_cache"; @@ -1587,6 +1589,21 @@ public class SessionVariable implements Serializable, Writable { ) public boolean enableBinarySearchFilteringPartitions = true; + @VariableMgr.VarAttr( + name = ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION, + flag = VariableMgr.GLOBAL, + description = { + "是否阻止后台动态分区任务自动删除分区。默认关闭。", + "Whether to prevent background dynamic partition tasks from automatically dropping " + + "partitions. OFF by default." + } + ) + private boolean enableDynamicPartitionDropProtection = false; + + public boolean isEnableDynamicPartitionDropProtection() { + return enableDynamicPartitionDropProtection; + } + @VariableMgr.VarAttr( name = CACHE_SORTED_PARTITION_INTERVAL_SECOND, fuzzy = false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index b0e328b4b82..03337be1fe3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -172,6 +172,36 @@ public class VariableMgr { return defaultSessionVariable; } + public static boolean isDynamicPartitionDropProtectionEnabled() { + rlock.lock(); + try { + return defaultSessionVariable.isEnableDynamicPartitionDropProtection(); + } finally { + rlock.unlock(); + } + } + + /** + * Sets a persistent global-only session variable while constructing an image. + * + * <p>The caller is responsible for persisting the resulting image. This method intentionally does not write an + * edit log because image construction may happen without a running edit log.</p> + */ + public static void setGlobalSessionVariableForImage(String name, String value) throws DdlException { + wlock.lock(); + try { + VarContext ctx = getVarContext(name); + Preconditions.checkArgument(ctx != null, "Unknown session variable: %s", name); + Preconditions.checkArgument(ctx.getObj() == defaultSessionVariable, + "Not a session variable: %s", name); + Preconditions.checkArgument((ctx.getFlag() & GLOBAL) != 0, + "Session variable is not global-only: %s", name); + setValue(defaultSessionVariable, new SessionVariableField(ctx.getField()), value); + } finally { + wlock.unlock(); + } + } + // Set value to a variable private static boolean setValue(Object obj, SessionVariableField sessionVariableField, String value) throws DdlException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index a9a818733d0..03394cbba4a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -30,19 +30,23 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.mysql.authenticate.TestLogAppender; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.qe.VariableMgr; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.utframe.UtFrameUtils; import com.google.common.collect.Lists; import com.google.common.collect.Range; +import org.apache.logging.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -133,6 +137,99 @@ public class DynamicPartitionTableTest { } } + private void setDynamicPartitionDropProtection(boolean enabled) throws DdlException { + VariableMgr.setGlobalSessionVariableForImage( + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION, Boolean.toString(enabled)); + } + + @Test + public void testDynamicPartitionDropProtection() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_drop_protection` (\n" + + " `k1` date NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "PARTITION BY RANGE (`k1`) ()\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"false\",\n" + + "\"dynamic_partition.start\" = \"-1\",\n" + + "\"dynamic_partition.end\" = \"1\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + createTable(createOlapTblStmt); + + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException("dynamic_partition_drop_protection"); + alterTable("ALTER TABLE test.dynamic_partition_drop_protection ADD PARTITION p_old " + + "VALUES [('2000-01-01'), ('2000-01-02'))"); + + try { + setDynamicPartitionDropProtection(true); + alterTable("ALTER TABLE test.dynamic_partition_drop_protection SET " + + "('dynamic_partition.enable' = 'true')"); + try (TestLogAppender appender = TestLogAppender.attach(DynamicPartitionScheduler.class, Level.WARN)) { + Env.getCurrentEnv().getDynamicPartitionScheduler() + .executeDynamicPartitionFirstTime(db.getId(), table.getId()); + Assert.assertTrue(appender.contains(Level.WARN, + "table [" + table.getId() + "-" + table.getName() + + "] because global session variable '" + + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION + "' is enabled")); + } + + Assert.assertTrue(table.getPartitionNames().contains("p_old")); + Assert.assertTrue("Dynamic partition creation should continue while drop protection is enabled", + table.getPartitionNames().size() > 1); + + setDynamicPartitionDropProtection(false); + Env.getCurrentEnv().getDynamicPartitionScheduler() + .executeDynamicPartitionFirstTime(db.getId(), table.getId()); + Assert.assertFalse(table.getPartitionNames().contains("p_old")); + } finally { + setDynamicPartitionDropProtection(false); + } + } + + @Test + public void testAutoPartitionRetentionDropProtection() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`auto_partition_drop_protection` (\n" + + " `k1` datetime NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "AUTO PARTITION BY RANGE (date_trunc(k1, 'day')) ()\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"partition.retention_count\" = \"1\"\n" + + ");"; + createTable(createOlapTblStmt); + + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException("auto_partition_drop_protection"); + alterTable("ALTER TABLE test.auto_partition_drop_protection ADD PARTITION p_old " + + "VALUES [('2000-01-01'), ('2000-01-02'))"); + alterTable("ALTER TABLE test.auto_partition_drop_protection ADD PARTITION p_new " + + "VALUES [('2020-01-01'), ('2020-01-02'))"); + + try { + setDynamicPartitionDropProtection(true); + Env.getCurrentEnv().getDynamicPartitionScheduler() + .executeDynamicPartitionFirstTime(db.getId(), table.getId()); + Assert.assertTrue(table.getPartitionNames().contains("p_old")); + Assert.assertTrue(table.getPartitionNames().contains("p_new")); + + setDynamicPartitionDropProtection(false); + Env.getCurrentEnv().getDynamicPartitionScheduler() + .executeDynamicPartitionFirstTime(db.getId(), table.getId()); + Assert.assertFalse(table.getPartitionNames().contains("p_old")); + Assert.assertTrue(table.getPartitionNames().contains("p_new")); + } finally { + setDynamicPartitionDropProtection(false); + } + } + @Test public void testNormal() throws Exception { String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java index 1bdd89b5e4f..5219437fdc1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java @@ -112,6 +112,32 @@ public class VariableMgrTest { Assert.fail("No exception throws."); } + @Test + public void testDynamicPartitionDropProtectionIsGlobalOnly() throws Exception { + try { + VariableMgr.setGlobalSessionVariableForImage( + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION, "false"); + Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled()); + + SetOptionsCommand sessionStmt = (SetOptionsCommand) UtFrameUtils.parseStmt( + "set enable_dynamic_partition_drop_protection=true", ctx); + Assertions.assertThrows(DdlException.class, () -> sessionStmt.run(ctx, null)); + + VariableMgr.setGlobalSessionVariableForImage( + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION, "true"); + Assertions.assertTrue(VariableMgr.isDynamicPartitionDropProtectionEnabled()); + Assertions.assertTrue(VariableMgr.newSessionVariable().isEnableDynamicPartitionDropProtection()); + + SetOptionsCommand globalStmt = (SetOptionsCommand) UtFrameUtils.parseStmt( + "set global enable_dynamic_partition_drop_protection=false", ctx); + globalStmt.run(ctx, null); + Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled()); + } finally { + VariableMgr.setGlobalSessionVariableForImage( + SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION, "false"); + } + } + @Test public void testAuthenticationPolicyIsTheReadOnlyMysqlDefault() throws Exception { SessionVariable sessionVariable = new SessionVariable(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
