This is an automated email from the ASF dual-hosted git repository.
alex-plekhanov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 1061e24cac9 IGNITE-28866 Fix deadlock on schema change with concurrent
node failure (#13333)
1061e24cac9 is described below
commit 1061e24cac910b5ffe76ab0a8ef2079fe3c1b069
Author: Aleksey Plekhanov <[email protected]>
AuthorDate: Thu Jul 9 18:41:11 2026 +0300
IGNITE-28866 Fix deadlock on schema change with concurrent node failure
(#13333)
Thank you for submitting the pull request to the Apache Ignite.
In order to streamline the review of the contribution
we ask you to ensure the following steps have been taken:
### The Contribution Checklist
- [ ] There is a single JIRA ticket related to the pull request.
- [ ] The web-link to the pull request is attached to the JIRA ticket.
- [ ] The JIRA ticket has the _Patch Available_ state.
- [ ] The pull request body describes changes that have been made.
The description explains _WHAT_ and _WHY_ was made instead of _HOW_.
- [ ] The pull request title is treated as the final commit message.
The following pattern must be used: `IGNITE-XXXX Change summary` where
`XXXX` - number of JIRA issue.
- [ ] A reviewer has been mentioned through the JIRA comments
(see [the Maintainers
list](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute#HowtoContribute-ReviewProcessandMaintainers))
- [ ] The pull request has been checked by the Teamcity Bot and
the `green visa` attached to the JIRA ticket (see tab `PR Check` at
[TC.Bot - Instance 1](https://tcbot2.sbt-ignite-dev.ru/prs.html) or
[TC.Bot - Instance 2](https://mtcga.gridgain.com/prs.html))
### Notes
- [How to
Contribute](https://cwiki.apache.org/confluence/display/IGNITE/How+to+Contribute)
- [Coding abbreviation
rules](https://cwiki.apache.org/confluence/display/IGNITE/Abbreviation+Rules)
- [Coding
Guidelines](https://cwiki.apache.org/confluence/display/IGNITE/Coding+Guidelines)
- [Apache Ignite Teamcity
Bot](https://cwiki.apache.org/confluence/display/IGNITE/Apache+Ignite+Teamcity+Bot)
If you need any help, please email [email protected] or ask anу
advice on http://asf.slack.com _#ignite_ channel.
---
.../processors/query/GridQueryProcessor.java | 14 ++++++++++
.../query/schema/SchemaOperationManager.java | 31 +++++++++++++---------
2 files changed, 33 insertions(+), 12 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 97cde3ad4ef..3008503660c 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -928,6 +928,7 @@ public class GridQueryProcessor extends
GridProcessorAdapter {
*
* @param schemaOp Schema operation.
*/
+ @SuppressWarnings("unchecked")
private void startSchemaChange(SchemaOperation schemaOp) {
assert Thread.holdsLock(stateMux);
assert !schemaOp.started();
@@ -986,6 +987,14 @@ public class GridQueryProcessor extends
GridProcessorAdapter {
mgr.start();
+ worker.future().listen(new IgniteInClosure<IgniteInternalFuture>() {
+ @Override public void apply(IgniteInternalFuture fut) {
+ synchronized (stateMux) {
+ mgr.onLocalNodeFinished(fut);
+ }
+ }
+ });
+
// Unwind pending IO messages.
if (!ctx.clientNode() && coordinator().isLocal())
unwindPendingMessages(schemaOp.id(), mgr);
@@ -4384,4 +4393,9 @@ public class GridQueryProcessor extends
GridProcessorAdapter {
public QueryEngine defaultQueryEngine() {
return dfltQryEngine;
}
+
+ /** */
+ public boolean isLockedByCurrentThread() {
+ return Thread.holdsLock(stateMux);
+ }
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaOperationManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaOperationManager.java
index 306c5caecf4..e4012cb52c6 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaOperationManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaOperationManager.java
@@ -30,7 +30,6 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.query.GridQueryProcessor;
import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.Nullable;
/**
@@ -49,7 +48,10 @@ public class SchemaOperationManager {
/** Operation handler. */
private final SchemaOperationWorker worker;
- /** Mutex for concurrency control. */
+ /**
+ * Mutex for concurrency control. It's crucial to maintain locks order to
avoid deadlocks,
+ * query processor should be locked before locking this mutex.
+ */
private final Object mux = new Object();
/** Participants. */
@@ -86,6 +88,8 @@ public class SchemaOperationManager {
this.qryProc = qryProc;
this.worker = worker;
+ assertQueryProcessorLockedBeforeOperationLock();
+
synchronized (mux) {
this.crd = crd;
@@ -96,17 +100,8 @@ public class SchemaOperationManager {
/**
* Map operation handling.
*/
- @SuppressWarnings("unchecked")
public void start() {
worker.start();
-
- synchronized (mux) {
- worker.future().listen(new IgniteInClosure<IgniteInternalFuture>()
{
- @Override public void apply(IgniteInternalFuture fut) {
- onLocalNodeFinished(fut);
- }
- });
- }
}
/**
@@ -114,7 +109,7 @@ public class SchemaOperationManager {
*
* @param fut Future.
*/
- private void onLocalNodeFinished(IgniteInternalFuture fut) {
+ public void onLocalNodeFinished(IgniteInternalFuture fut) {
assert fut.isDone();
if (ctx.clientNode())
@@ -131,6 +126,8 @@ public class SchemaOperationManager {
err = QueryUtils.wrapIfNeeded(e);
}
+ assertQueryProcessorLockedBeforeOperationLock();
+
synchronized (mux) {
if (isLocalCoordinator())
onNodeFinished(ctx.localNodeId(), err, worker.nop());
@@ -146,6 +143,8 @@ public class SchemaOperationManager {
* @param err Error.
*/
public void onNodeFinished(UUID nodeId, @Nullable SchemaOperationException
err, boolean nop) {
+ assertQueryProcessorLockedBeforeOperationLock();
+
synchronized (mux) {
assert isLocalCoordinator();
@@ -181,6 +180,8 @@ public class SchemaOperationManager {
* @param curCrd Current coordinator node.
*/
public void onNodeLeave(UUID nodeId, ClusterNode curCrd) {
+ assertQueryProcessorLockedBeforeOperationLock();
+
synchronized (mux) {
assert crd != null;
@@ -206,6 +207,12 @@ public class SchemaOperationManager {
}
}
+ /** */
+ private void assertQueryProcessorLockedBeforeOperationLock() {
+ assert qryProc.isLockedByCurrentThread() : "Locks order violation,
GridQueryProcessor.stateMux must be " +
+ "acquired before SchemaOperationManager.mux";
+ }
+
/**
* Check if operation finished.
*/
