This is an automated email from the ASF dual-hosted git repository. rmaucher pushed a commit to branch 9.0.x in repository https://gitbox.apache.org/repos/asf/tomcat.git
commit 5a1af0b2f2006293e87d7b62bd42abba1de5f431 Author: opencode <[email protected]> AuthorDate: Fri Oct 9 14:47:12 2026 +0200 Use short dedicated timeout for NonBlockingCoordinator merge liveness checks mergeOnArrive() verifies the members that an arriving coordination message carries but the local membership does not know yet with a raw TCP connect, sequentially, on the channel receiver worker threads. The connect timeout was the coordination message wait timeout, by default 15 seconds, so a token listing several stale or dead members, typically seen when a node returns from a partition, could pin receiver workers for a quarter of a minute and stall message processing, including acknowledgements, for all cluster connections. Verify the merging members with a separate mergeAliveTimeout property, defaulting to one second like the TcpFailureDetector connect timeout, and log a warning naming every member excluded from the merged view because it did not respond. Since the merge-time checks no longer use it, the coordination message wait timeout now only bounds the election wait. --- .../group/interceptors/LocalStrings.properties | 3 + .../group/interceptors/NonBlockingCoordinator.java | 67 +++++++++++++++++----- webapps/docs/changelog.xml | 9 +++ 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/java/org/apache/catalina/tribes/group/interceptors/LocalStrings.properties b/java/org/apache/catalina/tribes/group/interceptors/LocalStrings.properties index cd057b5e72..da8a19331f 100644 --- a/java/org/apache/catalina/tribes/group/interceptors/LocalStrings.properties +++ b/java/org/apache/catalina/tribes/group/interceptors/LocalStrings.properties @@ -72,6 +72,9 @@ nonBlockingCoordinator.heartbeat.inconsistency=Heartbeat found inconsistency, re nonBlockingCoordinator.memberAdded.failed=Unable to start election when member was added. nonBlockingCoordinator.memberAlive.failed=Unable to perform member alive check, assuming member down. nonBlockingCoordinator.memberDisappeared.failed=Unable to start election when member was removed. +nonBlockingCoordinator.mergeAliveTimeout.tooLarge=Merge liveness check timeout must be no more than [{0}] milliseconds +nonBlockingCoordinator.mergeAliveTimeout.tooSmall=Merge liveness check timeout must be at least 1 millisecond +nonBlockingCoordinator.mergeMemberNotAlive=Member [{0}] did not respond to the merge liveness check and is excluded from the merged view nonBlockingCoordinator.processCoordinationMessage.failed=Error processing coordination message. Could be fatal. nonBlockingCoordinator.report=CoordinationEvent[type={0}\n\ \tLocal: {1}\n\ diff --git a/java/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java b/java/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java index baada05d1f..57bf49a6eb 100644 --- a/java/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java +++ b/java/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java @@ -157,10 +157,17 @@ public class NonBlockingCoordinator extends ChannelInterceptorBase { * The wait blocks the calling thread, which for membership events is the channel * utility executor thread that also runs the channel heartbeat. Lowering this value * reduces the time that a single membership event can delay heartbeats and other - * membership processing. The same value is also used as the TCP connect timeout - * when verifying the liveness of members during a merge. + * membership processing. The liveness checks performed while merging an arriving + * coordination message use {@link #mergeAliveTimeout} instead. */ protected long waitForCoordMsgTimeout = 15000; + /** + * Time to wait for the TCP connect of the member liveness checks performed while + * merging an arriving coordination message. These checks run on the channel receiver + * worker threads, so this timeout is kept separate from and much shorter than + * {@link #waitForCoordMsgTimeout}. + */ + protected long mergeAliveTimeout = 1000; /** * Our current view */ @@ -212,8 +219,7 @@ public class NonBlockingCoordinator extends ChannelInterceptorBase { /** * Returns the time in milliseconds that a non-leader node waits for a coordination - * message when an election is started. The same value is also used as the TCP - * connect timeout when verifying the liveness of members during a merge. + * message when an election is started. * * @return the coordination message wait timeout in milliseconds */ @@ -221,14 +227,46 @@ public class NonBlockingCoordinator extends ChannelInterceptorBase { return waitForCoordMsgTimeout; } + /** + * Returns the time in milliseconds that the TCP connect of the member liveness + * checks waits during the merge of an arriving coordination message. + * + * @return the merge liveness check timeout in milliseconds + */ + public long getMergeAliveTimeout() { + return mergeAliveTimeout; + } + + /** + * Sets the time in milliseconds that the TCP connect of the member liveness checks + * waits during the merge of an arriving coordination message. Since these checks run + * sequentially on the channel receiver worker threads, the value should stay short to + * avoid pinning receiver workers when a coordination message carries stale or dead + * members. + * + * @param mergeAliveTimeout the merge liveness check timeout in milliseconds, at + * least one and no more than + * {@link Integer#MAX_VALUE} + */ + public void setMergeAliveTimeout(long mergeAliveTimeout) { + if (mergeAliveTimeout < 1) { + throw new IllegalArgumentException(sm.getString("nonBlockingCoordinator.mergeAliveTimeout.tooSmall")); + } + if (mergeAliveTimeout > Integer.MAX_VALUE) { + throw new IllegalArgumentException(sm.getString("nonBlockingCoordinator.mergeAliveTimeout.tooLarge", + Integer.toString(Integer.MAX_VALUE))); + } + this.mergeAliveTimeout = mergeAliveTimeout; + } + /** * Sets the time in milliseconds that a non-leader node waits for a coordination - * message when an election is started. The same value is also used as the TCP - * connect timeout when verifying the liveness of members during a merge. Since the - * wait blocks the calling thread, which for membership events is the single - * threaded channel utility executor that also runs the channel heartbeat, a lower - * value limits the delay that a single membership event can impose on other channel - * maintenance work. + * message when an election is started. Since the wait blocks the calling thread, + * which for membership events is the single threaded channel utility executor that + * also runs the channel heartbeat, a lower value limits the delay that a single + * membership event can impose on other channel maintenance work. The merge-time + * liveness checks are not affected by this value, they use + * {@link #getMergeAliveTimeout()}. * * @param waitForCoordMsgTimeout the coordination message wait timeout in * milliseconds, at least one and no more than @@ -399,14 +437,14 @@ public class NonBlockingCoordinator extends ChannelInterceptorBase { } /** - * Checks if a member is alive using the coordination message wait timeout - * ({@link #getWaitForCoordMsgTimeout()}) as the connection timeout. + * Checks if a member is alive using the merge liveness timeout + * ({@link #getMergeAliveTimeout()}) as the connection timeout. * * @param mbr The member to check * @return true if the member is alive */ protected boolean alive(Member mbr) { - return memberAlive(mbr, waitForCoordMsgTimeout); + return memberAlive(mbr, mergeAliveTimeout); } /** @@ -450,6 +488,9 @@ public class NonBlockingCoordinator extends ChannelInterceptorBase { Member[] diff = Arrays.diff(merged, membership, local); for (Member member : diff) { if (!alive(member)) { + if (log.isWarnEnabled()) { + log.warn(sm.getString("nonBlockingCoordinator.mergeMemberNotAlive", member)); + } merged.removeMember(member); } else { memberAdded(member, false); diff --git a/webapps/docs/changelog.xml b/webapps/docs/changelog.xml index 7d42710cfc..9e5ab11670 100644 --- a/webapps/docs/changelog.xml +++ b/webapps/docs/changelog.xml @@ -2473,6 +2473,15 @@ <code>maxQueueSize</code> limit eventually stopped being enforced. (remm) </fix> + <fix> + The <code>NonBlockingCoordinator</code> now uses a separate, + configurable <code>mergeAliveTimeout</code> with a default of + one second for the TCP liveness checks that run while merging + an arriving coordination message, instead of the 15 second + coordination message wait timeout. Members that do not respond + within this timeout are excluded from the merged view and the + exclusion is now logged. (remm) + </fix> </changelog> </subsection> <subsection name="Other"> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
