This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 78e07ed7776 branch-3.0: [opt](load) Add config to control commit lock
scope for tables #46996 (#47039)
78e07ed7776 is described below
commit 78e07ed77764fd3c79cda523b88bb3b47f401100
Author: Xin Liao <[email protected]>
AuthorDate: Thu Jan 16 10:42:04 2025 +0800
branch-3.0: [opt](load) Add config to control commit lock scope for tables
#46996 (#47039)
cherry pick from #46996
---
.../main/java/org/apache/doris/common/Config.java | 8 +++++
.../transaction/CloudGlobalTransactionMgr.java | 36 ++++++++++++++--------
2 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index cb70e6ccd1f..48bac41b511 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3187,6 +3187,14 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"})
public static int try_commit_lock_timeout_seconds = 5;
+ @ConfField(mutable = true, description = {"是否在事务提交时对所有表启用提交锁。设置为 true
时，所有表都会使用提交锁。"
+ + "设置为 false 时，仅对 Merge-On-Write 表使用提交锁。默认值为 true。",
+ "Whether to enable commit lock for all tables during transaction
commit."
+ + "If true, commit lock will be applied to all tables."
+ + "If false, commit lock will only be applied to Merge-On-Write
tables."
+ + "Default value is true." })
+ public static boolean enable_commit_lock_for_all_tables = true;
+
@ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"})
public static boolean enable_cloud_txn_lazy_commit = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 8ccbba707c7..52958a5d91b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -1079,6 +1079,21 @@ public class CloudGlobalTransactionMgr implements
GlobalTransactionMgrIface {
return true;
}
+ private List<Table> getTablesNeedCommitLock(List<Table> tableList) {
+ if (Config.enable_commit_lock_for_all_tables) {
+ // If enabled, lock all tables
+ return tableList.stream()
+ .sorted(Comparator.comparingLong(Table::getId))
+ .collect(Collectors.toList());
+ } else {
+ // If disabled, only lock MOW tables
+ return tableList.stream()
+ .filter(table -> table instanceof OlapTable &&
((OlapTable) table).getEnableUniqueKeyMergeOnWrite())
+ .sorted(Comparator.comparingLong(Table::getId))
+ .collect(Collectors.toList());
+ }
+ }
+
@Override
public boolean commitAndPublishTransaction(DatabaseIf db, List<Table>
tableList, long transactionId,
List<TabletCommitInfo>
tabletCommitInfos, long timeoutMillis,
@@ -1094,26 +1109,21 @@ public class CloudGlobalTransactionMgr implements
GlobalTransactionMgrIface {
}
}
- // Get tables that require commit lock - only MOW tables need this:
- // 1. Filter to keep only OlapTables with MOW enabled
- // 2. Sort by table ID to maintain consistent locking order and
prevent deadlocks
- List<Table> mowTableList = tableList.stream()
- .filter(table -> table instanceof OlapTable && ((OlapTable)
table).getEnableUniqueKeyMergeOnWrite())
- .sorted(Comparator.comparingLong(Table::getId))
- .collect(Collectors.toList());
- increaseWaitingLockCount(mowTableList);
- if (!MetaLockUtils.tryCommitLockTables(mowTableList, timeoutMillis,
TimeUnit.MILLISECONDS)) {
- decreaseWaitingLockCount(mowTableList);
+ List<Table> tablesToLock = getTablesNeedCommitLock(tableList);
+ increaseWaitingLockCount(tablesToLock);
+ if (!MetaLockUtils.tryCommitLockTables(tablesToLock, timeoutMillis,
TimeUnit.MILLISECONDS)) {
+ decreaseWaitingLockCount(tablesToLock);
// DELETE_BITMAP_LOCK_ERR will be retried on be
throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR,
"get table cloud commit lock timeout, tableList=("
- + StringUtils.join(mowTableList, ",") + ")");
+ + StringUtils.join(tablesToLock, ",") + ")");
}
+
try {
commitTransaction(db.getId(), tableList, transactionId,
tabletCommitInfos, txnCommitAttachment);
} finally {
- decreaseWaitingLockCount(mowTableList);
- MetaLockUtils.commitUnlockTables(mowTableList);
+ decreaseWaitingLockCount(tablesToLock);
+ MetaLockUtils.commitUnlockTables(tablesToLock);
}
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
