This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d562428b1d [enhancement](memory) reduce memory usage for failed broker
loads (#16974)
d562428b1d is described below
commit d562428b1d89c8aab433a06fa9d858ac6c50973b
Author: Zhengguo Yang <[email protected]>
AuthorDate: Fri Feb 24 12:07:02 2023 +0800
[enhancement](memory) reduce memory usage for failed broker loads (#16974)
Reduce more memory usage for failed broker load msg in fe after pr #15895
---
.../src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java | 5 ++++-
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 5 ++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index d61e21e98d..9514eecaf3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.UUID;
+import java.util.stream.Collectors;
public class LoadLoadingTask extends LoadTask {
private static final Logger LOG =
LogManager.getLogger(LoadLoadingTask.class);
@@ -177,7 +178,9 @@ public class LoadLoadingTask extends LoadTask {
curCoordinator.getLoadCounters(),
curCoordinator.getTrackingUrl(),
TabletCommitInfo.fromThrift(curCoordinator.getCommitInfos()),
-
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()));
+
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()
+
.stream().limit(Config.max_error_tablet_of_broker_load).collect(Collectors.toList())));
+ curCoordinator.getErrorTabletInfos().clear();
// Create profile of this task and add to the job profile.
createProfile(curCoordinator);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index f26eec4478..929e06f3f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -1125,7 +1125,10 @@ public class Coordinator {
private void updateErrorTabletInfos(List<TErrorTabletInfo>
errorTabletInfos) {
lock.lock();
try {
- this.errorTabletInfos.addAll(errorTabletInfos);
+ if (this.errorTabletInfos.size() <=
Config.max_error_tablet_of_broker_load) {
+
this.errorTabletInfos.addAll(errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load
+ -
this.errorTabletInfos.size()).collect(Collectors.toList()));
+ }
} finally {
lock.unlock();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
