Gary Tully created ARTEMIS-5693:
-----------------------------------
Summary: jgroups scaledown race with shared channel
Key: ARTEMIS-5693
URL: https://issues.apache.org/jira/browse/ARTEMIS-5693
Project: ActiveMQ Artemis
Issue Type: Bug
Components: Clustering
Affects Versions: 2.42.0
Reporter: Gary Tully
AMQ222181: Unable to scaleDown messages
logs:
{{
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
channelName='activemq_broadcast_channel', connected=true}::removeReceiver:
org.apache.activemq.artemis.api.core.jgroups.JGroupsReceiver@642857e0 on
activemq_broadcast_channel
java.lang.Exception: Trace
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.removeReceiver(JChannelWrapper.java:107)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:132)
at
org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
at
org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=0,
channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
channelName='activemq_broadcast_channel', connected=true}::RefCount-- 0 on
channel activemq_broadcast_channel
java.lang.Exception: Trace
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:87)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
at
org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
at
org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=0,
channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
channelName='activemq_broadcast_channel', connected=false}::Closing Channel:
activemq_broadcast_channel
java.lang.Exception: Trace
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.closeChannel(JChannelWrapper.java:100)
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.close(JChannelWrapper.java:90)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalCloseChannel(JGroupsBroadcastEndpoint.java:143)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.close(JGroupsBroadcastEndpoint.java:135)
at
org.apache.activemq.artemis.core.cluster.DiscoveryGroup.stop(DiscoveryGroup.java:168)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.doClose(ServerLocatorImpl.java:1426)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.close(ServerLocatorImpl.java:1402)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.closeLocator(ClusterConnectionImpl.java:517)
at
org.apache.activemq.artemis.core.server.cluster.impl.ClusterConnectionImpl.lambda$stop$0(ClusterConnectionImpl.java:506)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:59)
at
org.apache.activemq.artemis.utils.actors.OrderedExecutor.doTask(OrderedExecutor.java:32)
at
org.apache.activemq.artemis.utils.actors.ProcessorBase.executePendingTasks(ProcessorBase.java:69)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
org.apache.activemq.artemis.utils.ActiveMQThreadFactory$1.run(ActiveMQThreadFactory.java:120)
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
channel=managed-scaledown51a-ss-1-24689 (activemq_broadcast_channel),
channelName='activemq_broadcast_channel', connected=false}::RefCount++ = 1 on
channel activemq_broadcast_channel
AMQ221029: Stopped bridge
$.artemis.internal.sf.my-cluster.c9cfdf48-9f87-11f0-bd48-e24e2a7d3973
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper@2e645fbd{refCount=1,
channel=org.jgroups.JChannel@574a89e2,
channelName='activemq_broadcast_channel', connected=false}:: Connecting
activemq_broadcast_channel
java.lang.Exception: Trace
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:115)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
at
org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
at
org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
at
org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
at
org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
at
org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
...
at
org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:678)
at
org.eclipse.jetty.server.internal.HttpConnection.onFillable(HttpConnection.java:416)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1211)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
at java.base/java.lang.Thread.run(Thread.java:840)
AMQ222181: Unable to scaleDown messages
org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException: AMQ219004:
Failed to initialise session factory
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:288)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.createSessionFactory(ServerLocatorImpl.java:677)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:570)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:549)
at
org.apache.activemq.artemis.core.server.NodeLocator.connectToCluster(NodeLocator.java:85)
at
org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.connectToScaleDownTarget(PrimaryOnlyActivation.java:148)
at
org.apache.activemq.artemis.core.server.impl.PrimaryOnlyActivation.freezeConnections(PrimaryOnlyActivation.java:116)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.freezeConnections(ActiveMQServerImpl.java:1595)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1334)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:1251)
at
org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.fail(ActiveMQServerImpl.java:1230)
at
org.apache.activemq.artemis.core.management.impl.ActiveMQServerControlImpl.scaleDown(ActiveMQServerControlImpl.java:4209)
...
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: org.apache.activemq.artemis.api.core.ActiveMQInternalErrorException:
channel is closed
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:302)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.initialize(ServerLocatorImpl.java:284)
... 135 more
Caused by: java.lang.IllegalStateException: channel is closed
at org.jgroups.JChannel.checkClosed(JChannel.java:998)
at org.jgroups.JChannel._preConnect(JChannel.java:809)
at org.jgroups.JChannel.connect(JChannel.java:328)
at org.jgroups.JChannel.connect(JChannel.java:321)
at
org.apache.activemq.artemis.api.core.jgroups.JChannelWrapper.connect(JChannelWrapper.java:126)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.internalOpen(JGroupsBroadcastEndpoint.java:124)
at
org.apache.activemq.artemis.api.core.JGroupsBroadcastEndpoint.openClient(JGroupsBroadcastEndpoint.java:102)
at
org.apache.activemq.artemis.core.cluster.DiscoveryGroup.start(DiscoveryGroup.java:110)
at
org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.startDiscovery(ServerLocatorImpl.java:300)
... 136 more
}}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
For further information, visit: https://activemq.apache.org/contact
