This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 7a0a2c2a46 Solve the deadlock problem caused by queuing (#13191)
7a0a2c2a46 is described below
commit 7a0a2c2a46a0224d0c9fb1f649c1f901d11e814b
Author: sssqhai <[email protected]>
AuthorDate: Fri Dec 16 19:55:02 2022 +0800
Solve the deadlock problem caused by queuing (#13191)
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing,move the event to the tail
by throwing a exception
Co-authored-by: wfs <[email protected]>
---
...ntHandler.java => StateEventHandleFailure.java} | 25 ++++++++++------------
.../server/master/event/StateEventHandler.java | 3 ++-
.../event/TaskWaitTaskGroupStateHandler.java | 8 +++++--
.../master/runner/WorkflowExecuteRunnable.java | 8 +++++++
4 files changed, 27 insertions(+), 17 deletions(-)
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
similarity index 53%
copy from
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
copy to
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
index 93d4a7d4b3..5e757c7858 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
@@ -17,20 +17,17 @@
package org.apache.dolphinscheduler.server.master.event;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import
org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-
-public interface StateEventHandler {
+/**
+ * This exception represent the exception can be recovered, when we get this
exception,
+ * we will move the event to the fail of the queue.
+ */
+public class StateEventHandleFailure extends Exception {
- /**
- * Handle a event, if handle success will reture true, else return false
- *
- * @param stateEvent given state event.
- * @throws StateEventHandleException this exception means it can be
recovered.
- * @throws StateEventHandleError this exception means it cannot be
recovered, so the event need to drop.
- */
- boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) throws
StateEventHandleException, StateEventHandleError;
+ public StateEventHandleFailure(String message) {
+ super(message);
+ }
- StateEventType getEventType();
+ public StateEventHandleFailure(String message, Throwable throwable) {
+ super(message, throwable);
+ }
}
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
index 93d4a7d4b3..377ea71f62 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
@@ -28,9 +28,10 @@ public interface StateEventHandler {
* @param stateEvent given state event.
* @throws StateEventHandleException this exception means it can be
recovered.
* @throws StateEventHandleError this exception means it cannot be
recovered, so the event need to drop.
+ * @throws StateEventHandleException this means it can be recovered.
*/
boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) throws
StateEventHandleException, StateEventHandleError;
+ StateEvent stateEvent) throws
StateEventHandleException, StateEventHandleError, StateEventHandleFailure;
StateEventType getEventType();
}
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
index 02d08a6616..bb0c7b8b70 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
@@ -31,9 +31,13 @@ public class TaskWaitTaskGroupStateHandler implements
StateEventHandler {
private static final Logger logger =
LoggerFactory.getLogger(TaskWaitTaskGroupStateHandler.class);
@Override
- public boolean handleStateEvent(WorkflowExecuteRunnable
workflowExecuteRunnable, StateEvent stateEvent) {
+ public boolean handleStateEvent(WorkflowExecuteRunnable
workflowExecuteRunnable,
+ StateEvent stateEvent) throws
StateEventHandleFailure {
logger.info("Handle task instance wait task group event,
taskInstanceId: {}", stateEvent.getTaskInstanceId());
- return workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent);
+ if (!workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent)) {
+ throw new StateEventHandleFailure("Task state event handle failed
due to robing taskGroup resource failed");
+ }
+ return true;
}
@Override
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index efb4a944c7..ab0af2be5b 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -71,6 +71,7 @@ import
org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutor
import org.apache.dolphinscheduler.server.master.event.StateEvent;
import org.apache.dolphinscheduler.server.master.event.StateEventHandleError;
import
org.apache.dolphinscheduler.server.master.event.StateEventHandleException;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandleFailure;
import org.apache.dolphinscheduler.server.master.event.StateEventHandler;
import
org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager;
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
@@ -315,6 +316,13 @@ public class WorkflowExecuteRunnable implements
Callable<WorkflowSubmitStatue> {
stateEvent,
stateEventHandleException);
ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+ } catch (StateEventHandleFailure stateEventHandleFailure) {
+ logger.error("State event handle failed, will move event to
the tail: {}",
+ stateEvent,
+ stateEventHandleFailure);
+ this.stateEvents.remove(stateEvent);
+ this.stateEvents.offer(stateEvent);
+ ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (Exception e) {
// we catch the exception here, since if the state event
handle failed, the state event will still keep
// in the stateEvents queue.
