This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 399b43751f8 [fix](Export) fix dead lock of parallel Exporting (#43083)
399b43751f8 is described below
commit 399b43751f8acc4e7fef9c807cc254f66a4df13e
Author: Tiewei Fang <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Sat Nov 2 08:55:27 2024 +0800
[fix](Export) fix dead lock of parallel Exporting (#43083)
### What problem does this PR solve?
Problem Summary:
There are 2 locks, lock of ExportMgr and lock of ExportJob(synchronized
lock).
Previously, the lock order is wrong:
1. When cancelling job, it will first get job lock, then getting mgr
lock.
2. When removing old job, it will first get mgr lock, then getting job
lock.
This PR fix it by always getting job lock after mgr lock, to avoid dead
lock.
---
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java | 4 ++--
.../src/main/java/org/apache/doris/load/ExportJobStateTransfer.java | 5 ++---
fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java | 4 ++--
3 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 9841ae2b11d..0a07b5b6851 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -706,7 +706,7 @@ public class ExportJob implements Writable {
if (FeConstants.runningUnitTest) {
return;
}
- Env.getCurrentEnv().getEditLog().logExportUpdateState(id,
ExportJobState.CANCELLED);
+ Env.getCurrentEnv().getEditLog().logExportUpdateState(this,
ExportJobState.CANCELLED);
}
private void exportExportJob() {
@@ -749,7 +749,7 @@ public class ExportJob implements Writable {
outfileInfo = GsonUtils.GSON.toJson(allOutfileInfo);
// Clear the jobExecutorList to release memory.
jobExecutorList.clear();
- Env.getCurrentEnv().getEditLog().logExportUpdateState(id,
ExportJobState.FINISHED);
+ Env.getCurrentEnv().getEditLog().logExportUpdateState(this,
ExportJobState.FINISHED);
}
public void replayExportJobState(ExportJobState newState) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJobStateTransfer.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJobStateTransfer.java
index 06253b1f1e8..4b6f2ff9f51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJobStateTransfer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJobStateTransfer.java
@@ -54,10 +54,9 @@ public class ExportJobStateTransfer implements Writable {
}
// used for persisting one log
- public ExportJobStateTransfer(long jobId, ExportJobState state) {
- this.jobId = jobId;
+ public ExportJobStateTransfer(ExportJob job, ExportJobState state) {
+ this.jobId = job.getId();
this.state = state;
- ExportJob job = Env.getCurrentEnv().getExportMgr().getJob(jobId);
this.startTimeMs = job.getStartTimeMs();
this.finishTimeMs = job.getFinishTimeMs();
this.failMsg = job.getFailMsg();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index 5f94c95a249..83e7977a983 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -1621,8 +1621,8 @@ public class EditLog {
}
}
- public void logExportUpdateState(long jobId, ExportJobState newState) {
- ExportJobStateTransfer transfer = new ExportJobStateTransfer(jobId,
newState);
+ public void logExportUpdateState(ExportJob job, ExportJobState newState) {
+ ExportJobStateTransfer transfer = new ExportJobStateTransfer(job,
newState);
logEdit(OperationType.OP_EXPORT_UPDATE_STATE, transfer);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
