This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 27adb463b80 [chore](backup) limit the involved tablets in a backup job
#39987 (#40082)
27adb463b80 is described below
commit 27adb463b80428e737e8043d87d50bc4b4ff3955
Author: walter <[email protected]>
AuthorDate: Thu Aug 29 11:07:56 2024 +0800
[chore](backup) limit the involved tablets in a backup job #39987 (#40082)
cherry pick from #39987
---
.../src/main/java/org/apache/doris/common/Config.java | 9 +++++++++
.../src/main/java/org/apache/doris/backup/BackupJob.java | 11 +++++++++++
2 files changed, 20 insertions(+)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 44882f21768..fe61cc80343 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1472,6 +1472,15 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int max_backup_restore_job_num_per_db = 10;
+ /**
+ * Control the max num of tablets per backup job involved.
+ */
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "用于控制每次 backup job 允许备份的 tablet 上限，以避免 OOM",
+ "Control the max num of tablets per backup job involved, to avoid OOM"
+ })
+ public static int max_backup_tablets_per_job = 300000;
+
/**
* Control the default max num of the instance for a user.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 97c6e4930e0..5a94d82419e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -471,6 +471,17 @@ public class BackupJob extends AbstractJob {
}
}
+ // Limit the max num of tablets involved in a backup job, to avoid OOM.
+ if (unfinishedTaskIds.size() > Config.max_backup_tablets_per_job) {
+ String msg = String.format("the num involved tablets %d exceeds
the limit %d, "
+ + "which might cause the FE OOM, change config
`max_backup_tablets_per_job` "
+ + "to change this limitation",
+ unfinishedTaskIds.size(),
Config.max_backup_tablets_per_job);
+ LOG.warn(msg);
+ status = new Status(ErrCode.COMMON_ERROR, msg);
+ return;
+ }
+
backupMeta = new BackupMeta(copiedTables, copiedResources);
// send tasks
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
