[
https://issues.apache.org/jira/browse/IGNITE-25272?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Denis Chudov updated IGNITE-25272:
----------------------------------
Description:
As for now, it can be executed in threads like meta storage FSM and cause their
degradation. See thread dumps and logs in the attachment.
{code:java}
Thread
[name="%defaultNode%JRaft-FSMCaller-Disruptor-metastorage_group_stripe_0-0",
id=421, state=RUNNABLE, blockCnt=24898, waitCnt=36862]
at
[email protected]/java.lang.invoke.VarHandleReferences$FieldInstanceReadWrite.setRelease(VarHandleReferences.java:172)
at
[email protected]/java.lang.invoke.VarHandleGuards.guard_LL_V(VarHandleGuards.java:53)
at
[email protected]/java.util.concurrent.CompletableFuture.<init>(CompletableFuture.java:1975)
at
[email protected]/java.util.concurrent.CompletableFuture.failedFuture(CompletableFuture.java:2844)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.waitForActualState(PlacementDriverMessageProcessor.java:272)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.lambda$processLeaseGrantedMessage$6(PlacementDriverMessageProcessor.java:200)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor$$Lambda/0x00007f0a60d177b0.apply(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
[email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.lambda$processLeaseGrantedMessage$7(PlacementDriverMessageProcessor.java:172)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor$$Lambda/0x00007f0a60d17340.apply(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
[email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processLeaseGrantedMessage(PlacementDriverMessageProcessor.java:172)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processPlacementDriverMessage(PlacementDriverMessageProcessor.java:133)
at
app//org.apache.ignite.internal.replicator.ReplicaImpl.processPlacementDriverMessage(ReplicaImpl.java:134)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.lambda$onPlacementDriverMessageReceived$7(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a60d17108.apply(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
[email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.onPlacementDriverMessageReceived(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a602e2828.onReceived(Unknown
Source)
at
app//org.apache.ignite.internal.network.TrackableNetworkMessageHandler.onReceived(TrackableNetworkMessageHandler.java:42)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:385)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:313)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:242)
at
app//org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:190)
at
app//org.apache.ignite.internal.placementdriver.negotiation.LeaseNegotiator.negotiate(LeaseNegotiator.java:63)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater.lambda$updateLeaseBatchInternal$0(LeaseUpdater.java:596)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater$$Lambda/0x00007f0a60d15068.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$49(RaftGroupServiceImpl.java:645)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl$$Lambda/0x00007f0a6094a270.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:579)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.send0(DefaultMessagingService.java:267)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.respond(DefaultMessagingService.java:210)
at
app//org.apache.ignite.internal.network.MessagingService.respond(MessagingService.java:107)
at
app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:240)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor.sendResponse(ActionRequestProcessor.java:268)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor$1.result(ActionRequestProcessor.java:177)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$WriteCommandIterator$1.result(JraftServerImpl.java:995)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler$ResultCachingClosure.result(MetaStorageWriteHandler.java:453)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteWithTime(MetaStorageWriteHandler.java:217)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleNonCachedWriteCommand(MetaStorageWriteHandler.java:153)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteCommand(MetaStorageWriteHandler.java:123)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener$$Lambda/0x00007f0a609bf0d8.accept(Unknown
Source)
at
[email protected]/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener.onWrite(MetaStorageListener.java:193)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$DelegatingStateMachine.onApply(JraftServerImpl.java:825)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doApplyTasks(FSMCallerImpl.java:570)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doCommitted(FSMCallerImpl.java:536)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.runApplyTask(FSMCallerImpl.java:391)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:123)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:117)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.internalBatching(StripedDisruptor.java:359)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:308)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:279)
at
app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:167)
at
app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:122)
at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
at [email protected]/java.lang.Thread.run(Thread.java:1583){code}
and
{code:java}
Thread
[name="%defaultNode%JRaft-FSMCaller-Disruptor-metastorage_group_stripe_0-0",
id=421, state=RUNNABLE, blockCnt=24898, waitCnt=34991]
at
app//org.apache.ignite.internal.lang.IgniteStringFormatter.format(IgniteStringFormatter.java:126)
at
app//org.apache.ignite.internal.logger.IgniteLoggerImpl.logInternal(IgniteLoggerImpl.java:150)
at
app//org.apache.ignite.internal.logger.IgniteLoggerImpl.info(IgniteLoggerImpl.java:33)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processLeaseGrantedMessage(PlacementDriverMessageProcessor.java:169)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processPlacementDriverMessage(PlacementDriverMessageProcessor.java:133)
at
app//org.apache.ignite.internal.replicator.ReplicaImpl.processPlacementDriverMessage(ReplicaImpl.java:134)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.lambda$onPlacementDriverMessageReceived$7(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a60d17108.apply(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
[email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.onPlacementDriverMessageReceived(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a602e2828.onReceived(Unknown
Source)
at
app//org.apache.ignite.internal.network.TrackableNetworkMessageHandler.onReceived(TrackableNetworkMessageHandler.java:42)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:385)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:313)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:242)
at
app//org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:190)
at
app//org.apache.ignite.internal.placementdriver.negotiation.LeaseNegotiator.negotiate(LeaseNegotiator.java:63)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater.lambda$updateLeaseBatchInternal$0(LeaseUpdater.java:596)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater$$Lambda/0x00007f0a60d15068.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$49(RaftGroupServiceImpl.java:645)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl$$Lambda/0x00007f0a6094a270.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:579)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.send0(DefaultMessagingService.java:267)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.respond(DefaultMessagingService.java:210)
at
app//org.apache.ignite.internal.network.MessagingService.respond(MessagingService.java:107)
at
app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:240)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor.sendResponse(ActionRequestProcessor.java:268)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor$1.result(ActionRequestProcessor.java:177)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$WriteCommandIterator$1.result(JraftServerImpl.java:995)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler$ResultCachingClosure.result(MetaStorageWriteHandler.java:453)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteWithTime(MetaStorageWriteHandler.java:217)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleNonCachedWriteCommand(MetaStorageWriteHandler.java:153)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteCommand(MetaStorageWriteHandler.java:123)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener$$Lambda/0x00007f0a609bf0d8.accept(Unknown
Source)
at
[email protected]/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener.onWrite(MetaStorageListener.java:193)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$DelegatingStateMachine.onApply(JraftServerImpl.java:825)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doApplyTasks(FSMCallerImpl.java:570)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doCommitted(FSMCallerImpl.java:536)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.runApplyTask(FSMCallerImpl.java:454)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:123)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:117)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:322)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:279)
at
app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:167)
at
app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:122)
at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
at [email protected]/java.lang.Thread.run(Thread.java:1583){code}
was:
As for now, it can be executed in threads like meta storage FSM and cause their
degradation. See thread dumps and logs in the attachment.
{code:java}
Thread
[name="%defaultNode%JRaft-FSMCaller-Disruptor-metastorage_group_stripe_0-0",
id=421, state=RUNNABLE, blockCnt=24898, waitCnt=34991]
at
app//org.apache.ignite.internal.lang.IgniteStringFormatter.format(IgniteStringFormatter.java:126)
at
app//org.apache.ignite.internal.logger.IgniteLoggerImpl.logInternal(IgniteLoggerImpl.java:150)
at
app//org.apache.ignite.internal.logger.IgniteLoggerImpl.info(IgniteLoggerImpl.java:33)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processLeaseGrantedMessage(PlacementDriverMessageProcessor.java:169)
at
app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processPlacementDriverMessage(PlacementDriverMessageProcessor.java:133)
at
app//org.apache.ignite.internal.replicator.ReplicaImpl.processPlacementDriverMessage(ReplicaImpl.java:134)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.lambda$onPlacementDriverMessageReceived$7(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a60d17108.apply(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
[email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
at
app//org.apache.ignite.internal.replicator.ReplicaManager.onPlacementDriverMessageReceived(ReplicaManager.java:492)
at
app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a602e2828.onReceived(Unknown
Source)
at
app//org.apache.ignite.internal.network.TrackableNetworkMessageHandler.onReceived(TrackableNetworkMessageHandler.java:42)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:385)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:313)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:242)
at
app//org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:190)
at
app//org.apache.ignite.internal.placementdriver.negotiation.LeaseNegotiator.negotiate(LeaseNegotiator.java:63)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater.lambda$updateLeaseBatchInternal$0(LeaseUpdater.java:596)
at
app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater$$Lambda/0x00007f0a60d15068.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$49(RaftGroupServiceImpl.java:645)
at
app//org.apache.ignite.internal.raft.RaftGroupServiceImpl$$Lambda/0x00007f0a6094a270.accept(Unknown
Source)
at
[email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
[email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
[email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
[email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:579)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.send0(DefaultMessagingService.java:267)
at
app//org.apache.ignite.internal.network.DefaultMessagingService.respond(DefaultMessagingService.java:210)
at
app//org.apache.ignite.internal.network.MessagingService.respond(MessagingService.java:107)
at
app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:240)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor.sendResponse(ActionRequestProcessor.java:268)
at
app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor$1.result(ActionRequestProcessor.java:177)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$WriteCommandIterator$1.result(JraftServerImpl.java:995)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler$ResultCachingClosure.result(MetaStorageWriteHandler.java:453)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteWithTime(MetaStorageWriteHandler.java:217)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleNonCachedWriteCommand(MetaStorageWriteHandler.java:153)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteCommand(MetaStorageWriteHandler.java:123)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener$$Lambda/0x00007f0a609bf0d8.accept(Unknown
Source)
at
[email protected]/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener.onWrite(MetaStorageListener.java:193)
at
app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$DelegatingStateMachine.onApply(JraftServerImpl.java:825)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doApplyTasks(FSMCallerImpl.java:570)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doCommitted(FSMCallerImpl.java:536)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.runApplyTask(FSMCallerImpl.java:454)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:123)
at
app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:117)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:322)
at
app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:279)
at
app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:167)
at
app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:122)
at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
at [email protected]/java.lang.Thread.run(Thread.java:1583){code}
> Move the replica-side lease negotiation flow to dedicated thread pool
> ---------------------------------------------------------------------
>
> Key: IGNITE-25272
> URL: https://issues.apache.org/jira/browse/IGNITE-25272
> Project: Ignite
> Issue Type: Bug
> Reporter: Denis Chudov
> Assignee: Denis Chudov
> Priority: Major
> Labels: ignite-3
> Attachments: logs.tar.gz
>
>
> As for now, it can be executed in threads like meta storage FSM and cause
> their degradation. See thread dumps and logs in the attachment.
> {code:java}
> Thread
> [name="%defaultNode%JRaft-FSMCaller-Disruptor-metastorage_group_stripe_0-0",
> id=421, state=RUNNABLE, blockCnt=24898, waitCnt=36862]
> at
> [email protected]/java.lang.invoke.VarHandleReferences$FieldInstanceReadWrite.setRelease(VarHandleReferences.java:172)
> at
> [email protected]/java.lang.invoke.VarHandleGuards.guard_LL_V(VarHandleGuards.java:53)
> at
> [email protected]/java.util.concurrent.CompletableFuture.<init>(CompletableFuture.java:1975)
> at
> [email protected]/java.util.concurrent.CompletableFuture.failedFuture(CompletableFuture.java:2844)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.waitForActualState(PlacementDriverMessageProcessor.java:272)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.lambda$processLeaseGrantedMessage$6(PlacementDriverMessageProcessor.java:200)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor$$Lambda/0x00007f0a60d177b0.apply(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
> at
> [email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.lambda$processLeaseGrantedMessage$7(PlacementDriverMessageProcessor.java:172)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor$$Lambda/0x00007f0a60d17340.apply(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
> at
> [email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processLeaseGrantedMessage(PlacementDriverMessageProcessor.java:172)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processPlacementDriverMessage(PlacementDriverMessageProcessor.java:133)
> at
> app//org.apache.ignite.internal.replicator.ReplicaImpl.processPlacementDriverMessage(ReplicaImpl.java:134)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager.lambda$onPlacementDriverMessageReceived$7(ReplicaManager.java:492)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a60d17108.apply(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
> at
> [email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager.onPlacementDriverMessageReceived(ReplicaManager.java:492)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a602e2828.onReceived(Unknown
> Source)
> at
> app//org.apache.ignite.internal.network.TrackableNetworkMessageHandler.onReceived(TrackableNetworkMessageHandler.java:42)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:385)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:313)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:242)
> at
> app//org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:190)
> at
> app//org.apache.ignite.internal.placementdriver.negotiation.LeaseNegotiator.negotiate(LeaseNegotiator.java:63)
> at
> app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater.lambda$updateLeaseBatchInternal$0(LeaseUpdater.java:596)
> at
> app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater$$Lambda/0x00007f0a60d15068.accept(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
> at
> [email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
> at
> [email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> [email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
> at
> app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$49(RaftGroupServiceImpl.java:645)
> at
> app//org.apache.ignite.internal.raft.RaftGroupServiceImpl$$Lambda/0x00007f0a6094a270.accept(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
> at
> [email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
> at
> [email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> [email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:579)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.send0(DefaultMessagingService.java:267)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.respond(DefaultMessagingService.java:210)
> at
> app//org.apache.ignite.internal.network.MessagingService.respond(MessagingService.java:107)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:240)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor.sendResponse(ActionRequestProcessor.java:268)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor$1.result(ActionRequestProcessor.java:177)
> at
> app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$WriteCommandIterator$1.result(JraftServerImpl.java:995)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler$ResultCachingClosure.result(MetaStorageWriteHandler.java:453)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteWithTime(MetaStorageWriteHandler.java:217)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleNonCachedWriteCommand(MetaStorageWriteHandler.java:153)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteCommand(MetaStorageWriteHandler.java:123)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener$$Lambda/0x00007f0a609bf0d8.accept(Unknown
> Source)
> at
> [email protected]/java.util.Iterator.forEachRemaining(Iterator.java:133)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener.onWrite(MetaStorageListener.java:193)
> at
> app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$DelegatingStateMachine.onApply(JraftServerImpl.java:825)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doApplyTasks(FSMCallerImpl.java:570)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doCommitted(FSMCallerImpl.java:536)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.runApplyTask(FSMCallerImpl.java:391)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:123)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:117)
> at
> app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.internalBatching(StripedDisruptor.java:359)
> at
> app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:308)
> at
> app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:279)
> at
> app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:167)
> at
> app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:122)
> at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
> at [email protected]/java.lang.Thread.run(Thread.java:1583){code}
> and
> {code:java}
> Thread
> [name="%defaultNode%JRaft-FSMCaller-Disruptor-metastorage_group_stripe_0-0",
> id=421, state=RUNNABLE, blockCnt=24898, waitCnt=34991]
> at
> app//org.apache.ignite.internal.lang.IgniteStringFormatter.format(IgniteStringFormatter.java:126)
> at
> app//org.apache.ignite.internal.logger.IgniteLoggerImpl.logInternal(IgniteLoggerImpl.java:150)
> at
> app//org.apache.ignite.internal.logger.IgniteLoggerImpl.info(IgniteLoggerImpl.java:33)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processLeaseGrantedMessage(PlacementDriverMessageProcessor.java:169)
> at
> app//org.apache.ignite.internal.replicator.PlacementDriverMessageProcessor.processPlacementDriverMessage(PlacementDriverMessageProcessor.java:133)
> at
> app//org.apache.ignite.internal.replicator.ReplicaImpl.processPlacementDriverMessage(ReplicaImpl.java:134)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager.lambda$onPlacementDriverMessageReceived$7(ReplicaManager.java:492)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a60d17108.apply(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
> at
> [email protected]/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2341)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager.onPlacementDriverMessageReceived(ReplicaManager.java:492)
> at
> app//org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x00007f0a602e2828.onReceived(Unknown
> Source)
> at
> app//org.apache.ignite.internal.network.TrackableNetworkMessageHandler.onReceived(TrackableNetworkMessageHandler.java:42)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:385)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:313)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:242)
> at
> app//org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:190)
> at
> app//org.apache.ignite.internal.placementdriver.negotiation.LeaseNegotiator.negotiate(LeaseNegotiator.java:63)
> at
> app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater.lambda$updateLeaseBatchInternal$0(LeaseUpdater.java:596)
> at
> app//org.apache.ignite.internal.placementdriver.LeaseUpdater$Updater$$Lambda/0x00007f0a60d15068.accept(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
> at
> [email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
> at
> [email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> [email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
> at
> app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$49(RaftGroupServiceImpl.java:645)
> at
> app//org.apache.ignite.internal.raft.RaftGroupServiceImpl$$Lambda/0x00007f0a6094a270.accept(Unknown
> Source)
> at
> [email protected]/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
> at
> [email protected]/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
> at
> [email protected]/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
> at
> [email protected]/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2179)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:579)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.send0(DefaultMessagingService.java:267)
> at
> app//org.apache.ignite.internal.network.DefaultMessagingService.respond(DefaultMessagingService.java:210)
> at
> app//org.apache.ignite.internal.network.MessagingService.respond(MessagingService.java:107)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:240)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor.sendResponse(ActionRequestProcessor.java:268)
> at
> app//org.apache.ignite.raft.jraft.rpc.impl.ActionRequestProcessor$1.result(ActionRequestProcessor.java:177)
> at
> app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$WriteCommandIterator$1.result(JraftServerImpl.java:995)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler$ResultCachingClosure.result(MetaStorageWriteHandler.java:453)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteWithTime(MetaStorageWriteHandler.java:217)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleNonCachedWriteCommand(MetaStorageWriteHandler.java:153)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler.handleWriteCommand(MetaStorageWriteHandler.java:123)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener$$Lambda/0x00007f0a609bf0d8.accept(Unknown
> Source)
> at
> [email protected]/java.util.Iterator.forEachRemaining(Iterator.java:133)
> at
> app//org.apache.ignite.internal.metastorage.server.raft.MetaStorageListener.onWrite(MetaStorageListener.java:193)
> at
> app//org.apache.ignite.internal.raft.server.impl.JraftServerImpl$DelegatingStateMachine.onApply(JraftServerImpl.java:825)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doApplyTasks(FSMCallerImpl.java:570)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.doCommitted(FSMCallerImpl.java:536)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl.runApplyTask(FSMCallerImpl.java:454)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:123)
> at
> app//org.apache.ignite.raft.jraft.core.FSMCallerImpl$ApplyTaskHandler.onEvent(FSMCallerImpl.java:117)
> at
> app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:322)
> at
> app//org.apache.ignite.raft.jraft.disruptor.StripedDisruptor$StripeEntryHandler.onEvent(StripedDisruptor.java:279)
> at
> app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:167)
> at
> app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:122)
> at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
> at [email protected]/java.lang.Thread.run(Thread.java:1583){code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
