[ https://issues.apache.org/jira/browse/IGNITE-12981?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Maxim Muzafarov updated IGNITE-12981: ------------------------------------- Ignite Flags: (was: Docs Required,Release Notes Required) > Snapshot operation fails on exchange if coordinator left the cluster > -------------------------------------------------------------------- > > Key: IGNITE-12981 > URL: https://issues.apache.org/jira/browse/IGNITE-12981 > Project: Ignite > Issue Type: Bug > Reporter: Maxim Muzafarov > Assignee: Maxim Muzafarov > Priority: Major > Labels: iep-43 > Fix For: 2.9 > > > Possible steps: > 1. Start snapshot operation with {{SnapshotDicoveryMessage}} with triggering > pme-free. > 2. Stop the coordinator node. > 3. Fails on assertion (SingleMessage should not be sent). > {code} > [2020-05-03 > 15:05:35,548][ERROR][sys-#94%snapshot.IgniteClusterSnapshotSelfTest1%][IgniteTestResources] > Critical system error detected. Will be handled accordingly to configured > handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler > [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, > SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext > [type=CRITICAL_ERROR, err=java.lang.AssertionError: > GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryCustomEvent > [customMsg=null, affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > super=DiscoveryEvent [evtNode=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, > nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, > tstamp=1588507535430]], crd=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], > exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null, > affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > super=DiscoveryEvent [evtNode=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, > nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, > tstamp=1588507535430]], nodeId=ed2e276b, evt=DISCOVERY_CUSTOM_EVT], > added=true, exchangeType=ALL, initFut=GridFutureAdapter > [ignoreInterrupts=false, state=DONE, res=true, hash=1771643357], init=false, > lastVer=null, partReleaseFut=PartitionReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]], AtomicUpdateReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], > DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]], LocalTxReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], > AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]]]]]], exchActions=null, affChangeMsg=null, > centralizedAff=false, forceAffReassignment=false, exchangeLocE=null, > cacheChangeFailureMsgSent=false, done=true, state=SRV, > registerCachesFuture=null, partitionsSent=false, partitionsReceived=false, > delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter > [ignoreInterrupts=false, state=DONE, res=null, hash=197787912], > timeBag=o.a.i.i.util.TimeBag@1d97d94c, startTime=1365700834756, > initTime=1588507535439, rebalanced=false, evtLatch=0, remaining=HashSet > [ed2e276b-79a3-4de3-925a-68d602000000], mergedJoinExchMsgs=null, > awaitMergedMsgs=0, super=GridFutureAdapter [ignoreInterrupts=false, > state=DONE, res=AffinityTopologyVersion [topVer=3, minorTopVer=1], > hash=530592354]]]] > java.lang.AssertionError: GridDhtPartitionsExchangeFuture > [firstDiscoEvt=DiscoveryCustomEvent [customMsg=null, > affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > super=DiscoveryEvent [evtNode=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, > nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, > tstamp=1588507535430]], crd=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], > exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], discoEvt=DiscoveryCustomEvent [customMsg=null, > affTopVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > super=DiscoveryEvent [evtNode=TcpDiscoveryNode > [id=ed2e276b-79a3-4de3-925a-68d602000000, > consistentId=snapshot.IgniteClusterSnapshotSelfTest0, addrs=ArrayList > [127.0.0.1], sockAddrs=HashSet [/127.0.0.1:47500], discPort=47500, order=1, > intOrder=1, lastExchangeTime=1588507532715, loc=false, > ver=2.9.0#20200503-sha1:00000000, isClient=false], topVer=3, > nodeId8=f6fa6fad, msg=null, type=DISCOVERY_CUSTOM_EVT, > tstamp=1588507535430]], nodeId=ed2e276b, evt=DISCOVERY_CUSTOM_EVT], > added=true, exchangeType=ALL, initFut=GridFutureAdapter > [ignoreInterrupts=false, state=DONE, res=true, hash=1771643357], init=false, > lastVer=null, partReleaseFut=PartitionReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]], AtomicUpdateReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], > DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]], LocalTxReleaseFuture > [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], futures=[]], > AllTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > futures=[RemoteTxReleaseFuture [topVer=AffinityTopologyVersion [topVer=3, > minorTopVer=1], futures=[]]]]]], exchActions=null, affChangeMsg=null, > centralizedAff=false, forceAffReassignment=false, exchangeLocE=null, > cacheChangeFailureMsgSent=false, done=true, state=SRV, > registerCachesFuture=null, partitionsSent=false, partitionsReceived=false, > delayedLatestMsg=null, afterLsnrCompleteFut=GridFutureAdapter > [ignoreInterrupts=false, state=DONE, res=null, hash=197787912], > timeBag=org.apache.ignite.internal.util.TimeBag@1d97d94c, > startTime=1365700834756, initTime=1588507535439, rebalanced=false, > evtLatch=0, remaining=HashSet [ed2e276b-79a3-4de3-925a-68d602000000], > mergedJoinExchMsgs=null, awaitMergedMsgs=0, super=GridFutureAdapter > [ignoreInterrupts=false, state=DONE, res=AffinityTopologyVersion [topVer=3, > minorTopVer=1], hash=530592354]] > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2966) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$3.apply(GridDhtPartitionsExchangeFuture.java:2949) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.waitAndReplyToNode(GridDhtPartitionsExchangeFuture.java:2949) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processSingleMessage(GridDhtPartitionsExchangeFuture.java:3005) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$100(GridDhtPartitionsExchangeFuture.java:151) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2908) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2.apply(GridDhtPartitionsExchangeFuture.java:2896) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:354) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveSingleMessage(GridDhtPartitionsExchangeFuture.java:2896) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processSinglePartitionUpdate(GridCachePartitionExchangeManager.java:1999) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.preprocessSingleMessage(GridCachePartitionExchangeManager.java:532) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$1100(GridCachePartitionExchangeManager.java:182) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:415) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:397) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3705) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3684) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)