This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f1d37cb951e [improve](backup) Limit the output size of show
backup/restore job progress (#44681)
f1d37cb951e is described below
commit f1d37cb951ee4b83bc9e3ff1d6dca7fb5bf37ac5
Author: walter <[email protected]>
AuthorDate: Thu Nov 28 10:13:12 2024 +0800
[improve](backup) Limit the output size of show backup/restore job progress
(#44681)
A backup/restore job might involve millions of tablets, and the output
of the show backup/restore command might be huge without limitation.
---
.../java/org/apache/doris/alter/RollupJobV2.java | 4 ++--
.../java/org/apache/doris/backup/BackupJob.java | 22 ++++++++++++++++------
.../java/org/apache/doris/backup/RestoreJob.java | 22 ++++++++++++++++------
3 files changed, 34 insertions(+), 14 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 8eb87050bf6..1d09224e6fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -570,7 +570,7 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
continue;
}
- long visiableVersion = partition.getVisibleVersion();
+ long visibleVersion = partition.getVisibleVersion();
short expectReplicationNum =
tbl.getPartitionInfo().getReplicaAllocation(
partitionId).getTotalReplicaNum();
MaterializedIndex rollupIndex = entry.getValue();
@@ -579,7 +579,7 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
int healthyReplicaNum = 0;
for (Replica replica : replicas) {
if (!replica.isBad() && replica.getLastFailedVersion()
< 0
- &&
replica.checkVersionCatchUp(visiableVersion, false)) {
+ && replica.checkVersionCatchUp(visibleVersion,
false)) {
healthyReplicaNum++;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 621a2b1d9f7..d0a957dcd9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -1059,6 +1059,19 @@ public class BackupJob extends AbstractJob implements
GsonPostProcessable {
}
public synchronized List<String> getInfo() {
+ String unfinishedTaskIdsStr = unfinishedTaskIds.entrySet().stream()
+ .map(e -> "[" + e.getKey() + "=" + e.getValue() + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+ String taskProgressStr = taskProgress.entrySet().stream()
+ .map(e -> "[" + e.getKey() + ": " + e.getValue().first + "/" +
e.getValue().second + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+ String taskErrMsgStr = taskErrMsg.entrySet().stream()
+ .map(e -> "[" + e.getKey() + ": " + e.getValue() + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+
List<String> info = Lists.newArrayList();
info.add(String.valueOf(jobId));
info.add(label);
@@ -1069,12 +1082,9 @@ public class BackupJob extends AbstractJob implements
GsonPostProcessable {
info.add(TimeUtils.longToTimeString(snapshotFinishedTime));
info.add(TimeUtils.longToTimeString(snapshotUploadFinishedTime));
info.add(TimeUtils.longToTimeString(finishedTime));
- info.add(Joiner.on(", ").join(unfinishedTaskIds.entrySet()));
- info.add(Joiner.on(", ").join(taskProgress.entrySet().stream().map(
- e -> "[" + e.getKey() + ": " + e.getValue().first + "/" +
e.getValue().second + "]").collect(
- Collectors.toList())));
- info.add(Joiner.on(", ").join(taskErrMsg.entrySet().stream().map(n ->
"[" + n.getKey() + ": " + n.getValue()
- + "]").collect(Collectors.toList())));
+ info.add(unfinishedTaskIdsStr);
+ info.add(taskProgressStr);
+ info.add(taskErrMsgStr);
info.add(status.toString());
info.add(String.valueOf(timeoutMs / 1000));
return info;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 0b5b0cb0339..f1abb0c9e63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -2208,6 +2208,11 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
}
public synchronized List<String> getInfo(boolean isBrief) {
+ String unfinishedTaskIdsStr =
unfinishedSignatureToId.entrySet().stream()
+ .map(e -> "[" + e.getKey() + "=" + e.getValue() + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+
List<String> info = Lists.newArrayList();
info.add(String.valueOf(jobId));
info.add(label);
@@ -2227,13 +2232,18 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
info.add(TimeUtils.longToTimeString(snapshotFinishedTime));
info.add(TimeUtils.longToTimeString(downloadFinishedTime));
info.add(TimeUtils.longToTimeString(finishedTime));
- info.add(Joiner.on(", ").join(unfinishedSignatureToId.entrySet()));
+ info.add(unfinishedTaskIdsStr);
if (!isBrief) {
- info.add(Joiner.on(", ").join(taskProgress.entrySet().stream().map(
- e -> "[" + e.getKey() + ": " + e.getValue().first + "/" +
e.getValue().second + "]").collect(
- Collectors.toList())));
- info.add(Joiner.on(", ").join(taskErrMsg.entrySet().stream().map(n
-> "[" + n.getKey() + ": "
- + n.getValue() + "]").collect(Collectors.toList())));
+ String taskProgressStr = taskProgress.entrySet().stream()
+ .map(e -> "[" + e.getKey() + ": " + e.getValue().first +
"/" + e.getValue().second + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+ String taskErrMsgStr = taskErrMsg.entrySet().stream()
+ .map(n -> "[" + n.getKey() + ": " + n.getValue() + "]")
+ .limit(100)
+ .collect(Collectors.joining(", "));
+ info.add(taskProgressStr);
+ info.add(taskErrMsgStr);
}
info.add(status.toString());
info.add(String.valueOf(timeoutMs / 1000));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
