[ https://issues.apache.org/jira/browse/IGNITE-11852?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Pavel Kovalenko updated IGNITE-11852: ------------------------------------- Ignite Flags: (was: Docs Required) > Assertion errors when changing PME coordinator to locally joining node > ---------------------------------------------------------------------- > > Key: IGNITE-11852 > URL: https://issues.apache.org/jira/browse/IGNITE-11852 > Project: Ignite > Issue Type: Bug > Components: cache > Affects Versions: 2.5, 2.7 > Reporter: Pavel Kovalenko > Assignee: Pavel Kovalenko > Priority: Critical > Fix For: 2.8 > > Time Spent: 10m > Remaining Estimate: 0h > > When PME coordinator changed to locally joining node several assertion errors > may occur: > 1. When some other joining nodes finished PME: > {noformat} > [13:49:58] (err) Failed to notify listener: > o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1...@27296181java.lang.AssertionError: > AffinityTopologyVersion [topVer=2, minorTopVer=0] > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1546) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1535) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.lambda$forAllRegisteredCacheGroups$e0a6939d$1(CacheAffinitySharedManager.java:1281) > at > org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:10929) > at > org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:10831) > at > org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:10811) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.forAllRegisteredCacheGroups(CacheAffinitySharedManager.java:1280) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.onLocalJoin(CacheAffinitySharedManager.java:1535) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4189) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onBecomeCoordinator(GridDhtPartitionsExchangeFuture.java:4731) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$3400(GridDhtPartitionsExchangeFuture.java:145) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:4622) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:4611) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:398) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:510) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:489) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:466) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:281) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:143) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:44) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:398) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:510) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:489) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:455) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.InitNewCoordinatorFuture.onMessage(InitNewCoordinatorFuture.java:253) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveSingleMessage(GridDhtPartitionsExchangeFuture.java:2731) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processSinglePartitionUpdate(GridCachePartitionExchangeManager.java:1917) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$1300(GridCachePartitionExchangeManager.java:162) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:410) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:368) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3547) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3526) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1140) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:590) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:391) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:317) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:108) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:307) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1560) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1188) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$8.run(GridIoManager.java:1085) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > Suppressed: java.lang.AssertionError: AffinityTopologyVersion > [topVer=2, minorTopVer=0] > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1546) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1535) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.lambda$forAllRegisteredCacheGroups$e0a6939d$1(CacheAffinitySharedManager.java:1281) > at > org.apache.ignite.internal.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10910) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ... 3 more > Suppressed: java.lang.AssertionError: AffinityTopologyVersion > [topVer=2, minorTopVer=0] > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1546) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1535) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.lambda$forAllRegisteredCacheGroups$e0a6939d$1(CacheAffinitySharedManager.java:1281) > at > org.apache.ignite.internal.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10910) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ... 3 more > Suppressed: java.lang.AssertionError: AffinityTopologyVersion > [topVer=2, minorTopVer=0] > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1546) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager$11.applyx(CacheAffinitySharedManager.java:1535) > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.lambda$forAllRegisteredCacheGroups$e0a6939d$1(CacheAffinitySharedManager.java:1281) > at > org.apache.ignite.internal.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10910) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ... 3 more > Suppressed: java.lang.AssertionError: AffinityTopologyVersion > [topVer=2, minorTopVer=0] > ... 49 more > {noformat} > 2. When only client nodes finished PME: > {noformat} > [01:57:23] (err) Failed to notify listener: > o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$9$1$1...@166d555ejava.lang.AssertionError: > Requested and received affinity are different [requestedCnt=27, > receivedCnt=none, msg=GridDhtPartitionsFullMessage [parts=null, > partCntrs=null, partCntrs2=null, partHistSuppliers=null, partsToReload=null, > topVer=AffinityTopologyVersion [topVer=99, minorTopVer=0], errs=null, > resTopVer=AffinityTopologyVersion [topVer=99, minorTopVer=0], partCnt=0, > super=GridDhtPartitionsAbstractMessage [exchId=GridDhtPartitionExchangeId > [topVer=AffinityTopologyVersion [topVer=99, minorTopVer=0], > discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode > [id=213779fa-a9a0-47f7-b259-37c0483356e9, consistentId=node_1_1, > addrs=[172.25.1.48], sockAddrs=[lab48.gridgain.local/172.25.1.48:47500], > discPort=47500, order=99, intOrder=52, lastExchangeTime=1557010643241, > loc=true, ver=8.7.4#20190430-sha1:9dfd9fb2, isClient=false], topVer=99, > nodeId8=213779fa, msg=null, type=NODE_JOINED, tstamp=1557010643231], > nodeId=213779fa, evt=NODE_JOINED], lastVer=GridCacheVersion [topVer=0, > order=1557010642165, nodeOrder=0], super=GridCacheMessage [msgId=7305, > depInfo=null, lastAffChangedTopVer=AffinityTopologyVersion [topVer=99, > minorTopVer=0], err=null, skipPrepare=false]]]] > at > org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.onLocalJoin(CacheAffinitySharedManager.java:1527) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4096) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onBecomeCoordinator(GridDhtPartitionsExchangeFuture.java:4638) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$3500(GridDhtPartitionsExchangeFuture.java:141) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$9$1$1.apply(GridDhtPartitionsExchangeFuture.java:4529) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$9$1$1.apply(GridDhtPartitionsExchangeFuture.java:4518) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:394) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:348) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:336) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:506) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:485) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:462) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:284) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:143) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:44) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:394) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:348) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:336) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:506) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:485) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:451) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.InitNewCoordinatorFuture.onMessage(InitNewCoordinatorFuture.java:253) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveSingleMessage(GridDhtPartitionsExchangeFuture.java:2658) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processSinglePartitionUpdate(GridCachePartitionExchangeManager.java:1887) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$1300(GridCachePartitionExchangeManager.java:159) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:405) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$2.onMessage(GridCachePartitionExchangeManager.java:363) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3488) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3467) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1129) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:594) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:393) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:319) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1568) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1196) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1092) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > {noformat} -- This message was sent by Atlassian Jira (v8.3.4#803005)