This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new c49e88bf829 branch-4.1: pick some group commit prs(#60652)(#61555)
(#61953)
c49e88bf829 is described below
commit c49e88bf829935cd6b4faed9f44703f4a027cf75
Author: meiyi <[email protected]>
AuthorDate: Tue Mar 31 22:06:25 2026 +0800
branch-4.1: pick some group commit prs(#60652)(#61555) (#61953)
pick
1. Fix NPE in group commit when backend belongs to a different cluster
(https://github.com/apache/doris/pull/60652)
2. group commit select backend should consider isLoadAvailable
(https://github.com/apache/doris/pull/61555)
---------
Co-authored-by: deardeng <[email protected]>
---
.../org/apache/doris/load/GroupCommitManager.java | 30 ++++++++++++++++++----
.../main/java/org/apache/doris/qe/Coordinator.java | 26 +++++++++++++++++++
2 files changed, 51 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
index 25e72199d2a..bea5cd03c7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
@@ -213,7 +213,15 @@ public class GroupCommitManager {
try {
long backendId = new MasterOpExecutor(context)
.getGroupCommitLoadBeId(tableId, clusterName);
- return Env.getCurrentSystemInfo().getBackend(backendId);
+ Backend be = Env.getCurrentSystemInfo().getBackend(backendId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("selectBackendForGroupCommit on non-master:
tableId={}, clusterName={},"
+ + " backendId={}, backend={}, backendCluster={}",
+ tableId, clusterName, backendId,
+ be != null ? be.getHost() + ":" + be.getBePort() :
"null",
+ be != null ? be.getCloudClusterName() : "null");
+ }
+ return be;
} catch (Exception e) {
throw new LoadException(e.getMessage());
}
@@ -349,8 +357,7 @@ public class GroupCommitManager {
return null;
}
Backend backend =
Env.getCurrentSystemInfo().getBackend(backendId);
- if (backend != null && backend.isAlive() &&
!backend.isDecommissioned()
- && (!Config.isCloudMode() ||
!backend.isDecommissioning())) {
+ if (isBackendAvailable(backend, cluster)) {
return backend.getId();
} else {
tableToBeMap.remove(encode(cluster, tableId));
@@ -362,13 +369,26 @@ public class GroupCommitManager {
return null;
}
+ private boolean isBackendAvailable(Backend backend, String cluster) {
+ if (backend == null || !backend.isAlive() ||
backend.isDecommissioned() || !backend.isLoadAvailable()) {
+ return false;
+ }
+ if (!Config.isCloudMode()) {
+ return true;
+ }
+ // for cloud mode
+ if (backend.isDecommissioning()) {
+ return false;
+ }
+ return cluster == null ||
cluster.equals(backend.getCloudClusterName());
+ }
+
@Nullable
private Long getRandomBackend(String cluster, long tableId, List<Backend>
backends) {
OlapTable table = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getTableByTableId(tableId);
Collections.shuffle(backends);
for (Backend backend : backends) {
- if (backend.isAlive() && !backend.isDecommissioned() &&
(!Config.isCloudMode()
- || !backend.isDecommissioning())) {
+ if (isBackendAvailable(backend, cluster)) {
tableToBeMap.put(encode(cluster, tableId), backend.getId());
tableToPressureMap.put(tableId,
new
SlidingWindowCounter(table.getGroupCommitIntervalMs() / 1000 + 1));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index c2227b280b8..397ddfb982b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -567,6 +567,26 @@ public class Coordinator implements CoordInterface {
this.idToBackend =
Env.getCurrentSystemInfo().getBackendsByCurrentCluster();
+ // Log cluster info and groupCommitBackend for debugging NPE in
PipelineExecContext
+ if (groupCommitBackend != null) {
+ String currentCluster = "unknown";
+ try {
+ if (ConnectContext.get() != null) {
+ currentCluster = ConnectContext.get().getCloudCluster();
+ }
+ } catch (Exception e) {
+ LOG.debug("failed to get current cloud cluster for debug log",
e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("query {} prepare: currentCluster={},
idToBackend.size={}, idToBackend.keys={},"
+ + " groupCommitBackend=[id={}, host={},
cluster={}], groupCommitBackendInMap={}",
+ DebugUtil.printId(queryId), currentCluster,
idToBackend.size(), idToBackend.keySet(),
+ groupCommitBackend.getId(),
groupCommitBackend.getHost(),
+ groupCommitBackend.getCloudClusterName(),
+ idToBackend.containsKey(groupCommitBackend.getId()));
+ }
+ }
+
if (LOG.isDebugEnabled()) {
int backendNum = idToBackend.size();
StringBuilder backendInfos = new StringBuilder("backends info:");
@@ -838,6 +858,12 @@ public class Coordinator implements CoordInterface {
// So that we can use one RPC to send all fragment instances
of a BE.
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry
: tParams.entrySet()) {
Long backendId =
this.addressToBackendID.get(entry.getKey());
+ if (backendId == null) {
+ LOG.warn("query {} sendPipelineCtx: addressToBackendID
lookup returned null!"
+ + " address={}, fragmentId={},
addressToBackendID={}",
+ DebugUtil.printId(queryId), entry.getKey(),
+ fragment.getFragmentId(), addressToBackendID);
+ }
backendFragments.add(Pair.of(fragment.getFragmentId(),
backendId));
PipelineExecContext pipelineExecContext = new
PipelineExecContext(fragment.getFragmentId(),
entry.getValue(), idToBackend.get(backendId),
executionProfile, jobId);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]